# Updated by Wei on 2014/02/05 night at school

from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

print "Program Begins..."

NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES = 190892

print "Step1: Load the Postings into the heap1:"
# when fill1PercentDictFlag == False, stop filling the dict
# when fill1PercentDictFlag == True, continue to fill the dict
fill1PercentDictFlag = True
fill5PercentDictFlag = True
fill10PercentDictFlag = True
fill20PercentDictFlag = True
fill30PercentDictFlag = True
fill40PercentDictFlag = True
fill50PercentDictFlag = True
fill60PercentDictFlag = True
fill70PercentDictFlag = True
fill80PercentDictFlag = True
fill90PercentDictFlag = True
fill100PercentDictFlag = True

top10PostingDictPoppedAT1PercentFromFile1 = {}
top10PostingDictPoppedAT5PercentFromFile1 = {}
top10PostingDictPoppedAT10PercentFromFile1 = {}
top10PostingDictPoppedAT20PercentFromFile1 = {}
top10PostingDictPoppedAT30PercentFromFile1 = {}
top10PostingDictPoppedAT40PercentFromFile1 = {}
top10PostingDictPoppedAT50PercentFromFile1 = {}
top10PostingDictPoppedAT60PercentFromFile1 = {}
top10PostingDictPoppedAT70PercentFromFile1 = {}
top10PostingDictPoppedAT80PercentFromFile1 = {}
top10PostingDictPoppedAT90PercentFromFile1 = {}
top10PostingDictPoppedAT100PercentFromFile1 = {}

# step1:
# option1 weight = 0:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131122Afternoon_CORRECT"
# option2 weight = 1:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131121Night_CORRECT"
# option3 weight = 1000:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131122Night_CORRECT"
# Updated by Wei on 2014/02/05 night at school
inputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/debug_20140205Night_weight_0"

inputFileHandler = open(inputFileName,"r")
previousAccumulatedNumOfPostingCounter = 0
currentNumOfPostingCounter = 0
for line in inputFileHandler.readlines():
    if line.strip().startswith("1%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill1PercentDictFlag = False
        print "1%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("5%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill5PercentDictFlag = False
        print "5%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("10%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill10PercentDictFlag = False
        print "10%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("20%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill20PercentDictFlag = False
        print "20%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("30%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill30PercentDictFlag = False
        print "30%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("40%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill40PercentDictFlag = False
        print "40%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("50%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill50PercentDictFlag = False
        print "50%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("60%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill60PercentDictFlag = False
        print "60%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("70%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill70PercentDictFlag = False
        print "70%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)  
    if line.strip().startswith("80%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill80PercentDictFlag = False
        print "80%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("90%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill90PercentDictFlag = False
        print "90%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("100%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill100PercentDictFlag = False
        print "100%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)    
    if line.strip().startswith("Posting_ID:"):
        # print "line:",line 
        currentNumOfPostingCounter += 1
        lineElements = line.strip().split(" ")
        termIDInStringFormat = lineElements[-2]
        docIDInStringFormat = lineElements[-1]
        currentPostingKey = termIDInStringFormat + "_" + docIDInStringFormat
        # flags check follows:
        if fill1PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT1PercentFromFile1:
                top10PostingDictPoppedAT1PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark1"
                print "currentPostingKey:",currentPostingKey
                exit(1)
        if fill5PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT5PercentFromFile1:
                top10PostingDictPoppedAT5PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark2"
                exit(1)
        if fill10PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT10PercentFromFile1:
                top10PostingDictPoppedAT10PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark3"
                exit(1)
        if fill20PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT20PercentFromFile1:
                top10PostingDictPoppedAT20PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark4"
                exit(1)
        if fill30PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT30PercentFromFile1:
                top10PostingDictPoppedAT30PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark5"
                exit(1)
        if fill40PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT40PercentFromFile1:
                top10PostingDictPoppedAT40PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark6"
                exit(1)        
        if fill50PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT50PercentFromFile1:
                top10PostingDictPoppedAT50PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark7"
                exit(1)
        if fill60PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT60PercentFromFile1:
                top10PostingDictPoppedAT60PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark8"
                exit(1)
        if fill70PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT70PercentFromFile1:
                top10PostingDictPoppedAT70PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark9"
                exit(1)
        if fill80PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT80PercentFromFile1:
                top10PostingDictPoppedAT80PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark10"
                exit(1)
        if fill90PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT90PercentFromFile1:
                top10PostingDictPoppedAT90PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark11"
                exit(1)
        if fill100PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT100PercentFromFile1:
                top10PostingDictPoppedAT100PercentFromFile1[currentPostingKey] = 1
            else:
                print "mark12"
                exit(1)
inputFileHandler.close()



print "Overall Processing Statistics:"
print "len(top10PostingDictPopped @ 1%): ",len(top10PostingDictPoppedAT1PercentFromFile1)
print "len(top10PostingDictPopped @ 5%): ",len(top10PostingDictPoppedAT5PercentFromFile1)
print "len(top10PostingDictPopped @ 10%): ",len(top10PostingDictPoppedAT10PercentFromFile1)
print "len(top10PostingDictPopped @ 20%): ",len(top10PostingDictPoppedAT20PercentFromFile1)
print "len(top10PostingDictPopped @ 30%): ",len(top10PostingDictPoppedAT30PercentFromFile1)
print "len(top10PostingDictPopped @ 40%): ",len(top10PostingDictPoppedAT40PercentFromFile1)
print "len(top10PostingDictPopped @ 50%): ",len(top10PostingDictPoppedAT50PercentFromFile1)
print "len(top10PostingDictPopped @ 60%): ",len(top10PostingDictPoppedAT60PercentFromFile1)
print "len(top10PostingDictPopped @ 70%): ",len(top10PostingDictPoppedAT70PercentFromFile1)
print "len(top10PostingDictPopped @ 80%): ",len(top10PostingDictPoppedAT80PercentFromFile1)
print "len(top10PostingDictPopped @ 90%): ",len(top10PostingDictPoppedAT90PercentFromFile1)
print "len(top10PostingDictPopped @ 100%): ",len(top10PostingDictPoppedAT100PercentFromFile1)


top10PostingDictPoppedAT1PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT1PercentFromFile1)
top10PostingDictPoppedAT5PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT5PercentFromFile1)
top10PostingDictPoppedAT10PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT10PercentFromFile1)
top10PostingDictPoppedAT20PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT20PercentFromFile1)
top10PostingDictPoppedAT30PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT30PercentFromFile1)
top10PostingDictPoppedAT40PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT40PercentFromFile1)
top10PostingDictPoppedAT50PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT50PercentFromFile1)
top10PostingDictPoppedAT60PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT60PercentFromFile1)
top10PostingDictPoppedAT70PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT70PercentFromFile1)
top10PostingDictPoppedAT80PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT80PercentFromFile1)
top10PostingDictPoppedAT90PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT90PercentFromFile1)
top10PostingDictPoppedAT100PercentInSetFormatFromFile1 = set(top10PostingDictPoppedAT100PercentFromFile1)

print "Step2: Load the Postings into the heap2:"

# when fill1PercentDictFlag == False, stop filling the dict
# when fill1PercentDictFlag == True, continue to fill the dict
fill1PercentDictFlag = True
fill5PercentDictFlag = True
fill10PercentDictFlag = True
fill20PercentDictFlag = True
fill30PercentDictFlag = True
fill40PercentDictFlag = True
fill50PercentDictFlag = True
fill60PercentDictFlag = True
fill70PercentDictFlag = True
fill80PercentDictFlag = True
fill90PercentDictFlag = True
fill100PercentDictFlag = True

top10PostingDictPoppedAT1PercentFromFile2 = {}
top10PostingDictPoppedAT5PercentFromFile2 = {}
top10PostingDictPoppedAT10PercentFromFile2 = {}
top10PostingDictPoppedAT20PercentFromFile2 = {}
top10PostingDictPoppedAT30PercentFromFile2 = {}
top10PostingDictPoppedAT40PercentFromFile2 = {}
top10PostingDictPoppedAT50PercentFromFile2 = {}
top10PostingDictPoppedAT60PercentFromFile2 = {}
top10PostingDictPoppedAT70PercentFromFile2 = {}
top10PostingDictPoppedAT80PercentFromFile2 = {}
top10PostingDictPoppedAT90PercentFromFile2 = {}
top10PostingDictPoppedAT100PercentFromFile2 = {}

# step1:
# option1 weight = 0:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131122Afternoon_CORRECT"
# option2 weight = 1:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131121Night_CORRECT"
# option3 weight = 1000:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131122Night_CORRECT"
# Updated by Wei on 2014/02/05 night at school
inputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/debug_20140205Night_weight_10"

inputFileHandler = open(inputFileName,"r")
previousAccumulatedNumOfPostingCounter = 0
currentNumOfPostingCounter = 0
for line in inputFileHandler.readlines():
    if line.strip().startswith("1%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill1PercentDictFlag = False
        print "1%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("5%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill5PercentDictFlag = False
        print "5%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("10%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill10PercentDictFlag = False
        print "10%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("20%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill20PercentDictFlag = False
        print "20%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("30%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill30PercentDictFlag = False
        print "30%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("40%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill40PercentDictFlag = False
        print "40%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("50%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill50PercentDictFlag = False
        print "50%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("60%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill60PercentDictFlag = False
        print "60%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("70%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill70PercentDictFlag = False
        print "70%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)  
    if line.strip().startswith("80%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill80PercentDictFlag = False
        print "80%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("90%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill90PercentDictFlag = False
        print "90%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("100%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill100PercentDictFlag = False
        print "100%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)    
    if line.strip().startswith("Posting_ID:"):
        # print "line:",line 
        currentNumOfPostingCounter += 1
        lineElements = line.strip().split(" ")
        termIDInStringFormat = lineElements[-2]
        docIDInStringFormat = lineElements[-1]
        currentPostingKey = termIDInStringFormat + "_" + docIDInStringFormat
        # flags check follows:
        if fill1PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT1PercentFromFile2:
                top10PostingDictPoppedAT1PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark1"
                print "currentPostingKey:",currentPostingKey
                exit(1)
        if fill5PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT5PercentFromFile2:
                top10PostingDictPoppedAT5PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark2"
                exit(1)
        if fill10PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT10PercentFromFile2:
                top10PostingDictPoppedAT10PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark3"
                exit(1)
        if fill20PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT20PercentFromFile2:
                top10PostingDictPoppedAT20PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark4"
                exit(1)
        if fill30PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT30PercentFromFile2:
                top10PostingDictPoppedAT30PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark5"
                exit(1)
        if fill40PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT40PercentFromFile2:
                top10PostingDictPoppedAT40PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark6"
                exit(1)        
        if fill50PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT50PercentFromFile2:
                top10PostingDictPoppedAT50PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark7"
                exit(1)
        if fill60PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT60PercentFromFile2:
                top10PostingDictPoppedAT60PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark8"
                exit(1)
        if fill70PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT70PercentFromFile2:
                top10PostingDictPoppedAT70PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark9"
                exit(1)
        if fill80PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT80PercentFromFile2:
                top10PostingDictPoppedAT80PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark10"
                exit(1)
        if fill90PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT90PercentFromFile2:
                top10PostingDictPoppedAT90PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark11"
                exit(1)
        if fill100PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT100PercentFromFile2:
                top10PostingDictPoppedAT100PercentFromFile2[currentPostingKey] = 1
            else:
                print "mark12"
                exit(1)
inputFileHandler.close()



print "Overall Processing Statistics:"
print "len(top10PostingDictPopped @ 1%): ",len(top10PostingDictPoppedAT1PercentFromFile2)
print "len(top10PostingDictPopped @ 5%): ",len(top10PostingDictPoppedAT5PercentFromFile2)
print "len(top10PostingDictPopped @ 10%): ",len(top10PostingDictPoppedAT10PercentFromFile2)
print "len(top10PostingDictPopped @ 20%): ",len(top10PostingDictPoppedAT20PercentFromFile2)
print "len(top10PostingDictPopped @ 30%): ",len(top10PostingDictPoppedAT30PercentFromFile2)
print "len(top10PostingDictPopped @ 40%): ",len(top10PostingDictPoppedAT40PercentFromFile2)
print "len(top10PostingDictPopped @ 50%): ",len(top10PostingDictPoppedAT50PercentFromFile2)
print "len(top10PostingDictPopped @ 60%): ",len(top10PostingDictPoppedAT60PercentFromFile2)
print "len(top10PostingDictPopped @ 70%): ",len(top10PostingDictPoppedAT70PercentFromFile2)
print "len(top10PostingDictPopped @ 80%): ",len(top10PostingDictPoppedAT80PercentFromFile2)
print "len(top10PostingDictPopped @ 90%): ",len(top10PostingDictPoppedAT90PercentFromFile2)
print "len(top10PostingDictPopped @ 100%): ",len(top10PostingDictPoppedAT100PercentFromFile2)
print

top10PostingDictPoppedAT1PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT1PercentFromFile2)
top10PostingDictPoppedAT5PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT5PercentFromFile2)
top10PostingDictPoppedAT10PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT10PercentFromFile2)
top10PostingDictPoppedAT20PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT20PercentFromFile2)
top10PostingDictPoppedAT30PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT30PercentFromFile2)
top10PostingDictPoppedAT40PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT40PercentFromFile2)
top10PostingDictPoppedAT50PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT50PercentFromFile2)
top10PostingDictPoppedAT60PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT60PercentFromFile2)
top10PostingDictPoppedAT70PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT70PercentFromFile2)
top10PostingDictPoppedAT80PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT80PercentFromFile2)
top10PostingDictPoppedAT90PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT90PercentFromFile2)
top10PostingDictPoppedAT100PercentInSetFormatFromFile2 = set(top10PostingDictPoppedAT100PercentFromFile2)

intersectionSet = top10PostingDictPoppedAT1PercentInSetFormatFromFile1.intersection( top10PostingDictPoppedAT1PercentInSetFormatFromFile2 )
unionSet = top10PostingDictPoppedAT1PercentInSetFormatFromFile1.union( top10PostingDictPoppedAT1PercentInSetFormatFromFile2 )
print "1%: ",len(intersectionSet),len(unionSet),len(intersectionSet)/len(unionSet)
intersectionSet = top10PostingDictPoppedAT5PercentInSetFormatFromFile1.intersection( top10PostingDictPoppedAT5PercentInSetFormatFromFile2 )
unionSet = top10PostingDictPoppedAT5PercentInSetFormatFromFile1.union( top10PostingDictPoppedAT5PercentInSetFormatFromFile2 )
print "5%: ",len(intersectionSet),len(unionSet),len(intersectionSet)/len(unionSet)
intersectionSet = top10PostingDictPoppedAT10PercentInSetFormatFromFile1.intersection( top10PostingDictPoppedAT10PercentInSetFormatFromFile2 )
unionSet = top10PostingDictPoppedAT10PercentInSetFormatFromFile1.union( top10PostingDictPoppedAT10PercentInSetFormatFromFile2 )
print "10%: ",len(intersectionSet),len(unionSet),len(intersectionSet)/len(unionSet)
intersectionSet = top10PostingDictPoppedAT20PercentInSetFormatFromFile1.intersection( top10PostingDictPoppedAT20PercentInSetFormatFromFile2 )
unionSet = top10PostingDictPoppedAT20PercentInSetFormatFromFile1.union( top10PostingDictPoppedAT20PercentInSetFormatFromFile2 )
print "20%: ",len(intersectionSet),len(unionSet),len(intersectionSet)/len(unionSet)
intersectionSet = top10PostingDictPoppedAT30PercentInSetFormatFromFile1.intersection( top10PostingDictPoppedAT30PercentInSetFormatFromFile2 )
unionSet = top10PostingDictPoppedAT30PercentInSetFormatFromFile1.union( top10PostingDictPoppedAT30PercentInSetFormatFromFile2 )
print "30%: ",len(intersectionSet),len(unionSet),len(intersectionSet)/len(unionSet)
intersectionSet = top10PostingDictPoppedAT40PercentInSetFormatFromFile1.intersection( top10PostingDictPoppedAT40PercentInSetFormatFromFile2 )
unionSet = top10PostingDictPoppedAT40PercentInSetFormatFromFile1.union( top10PostingDictPoppedAT40PercentInSetFormatFromFile2 )
print "40%: ",len(intersectionSet),len(unionSet),len(intersectionSet)/len(unionSet)
intersectionSet = top10PostingDictPoppedAT50PercentInSetFormatFromFile1.intersection( top10PostingDictPoppedAT50PercentInSetFormatFromFile2 )
unionSet = top10PostingDictPoppedAT50PercentInSetFormatFromFile1.union( top10PostingDictPoppedAT50PercentInSetFormatFromFile2 )
print "50%: ",len(intersectionSet),len(unionSet),len(intersectionSet)/len(unionSet)


'''
print "len(top10PostingDictPoppedAT1PercentFromFile1):",len(top10PostingDictPoppedAT1PercentFromFile1)
print "len(top10PostingDictPoppedAT5PercentFromFile1):",len(top10PostingDictPoppedAT5PercentFromFile1)
print "len(top10PostingDictPoppedAT10PercentFromFile1):",len(top10PostingDictPoppedAT10PercentFromFile1)
print "len(top10PostingDictPoppedAT20PercentFromFile1):",len(top10PostingDictPoppedAT20PercentFromFile1)
print "len(top10PostingDictPoppedAT30PercentFromFile1):",len(top10PostingDictPoppedAT30PercentFromFile1)
print "len(top10PostingDictPoppedAT40PercentFromFile1):",len(top10PostingDictPoppedAT40PercentFromFile1)
print "len(top10PostingDictPoppedAT50PercentFromFile1):",len(top10PostingDictPoppedAT50PercentFromFile1)
print "len(top10PostingDictPoppedAT60PercentFromFile1):",len(top10PostingDictPoppedAT60PercentFromFile1)
print "len(top10PostingDictPoppedAT70PercentFromFile1):",len(top10PostingDictPoppedAT70PercentFromFile1)
print "len(top10PostingDictPoppedAT80PercentFromFile1):",len(top10PostingDictPoppedAT80PercentFromFile1)
print "len(top10PostingDictPoppedAT90PercentFromFile1):",len(top10PostingDictPoppedAT90PercentFromFile1)
print "len(top10PostingDictPoppedAT100PercentFromFile1):",len(top10PostingDictPoppedAT100PercentFromFile1)

print "len(top10PostingDictPoppedAT1PercentFromFile2):",len(top10PostingDictPoppedAT1PercentFromFile2)
print "len(top10PostingDictPoppedAT5PercentFromFile2):",len(top10PostingDictPoppedAT5PercentFromFile2)
print "len(top10PostingDictPoppedAT10PercentFromFile2):",len(top10PostingDictPoppedAT10PercentFromFile2)
print "len(top10PostingDictPoppedAT20PercentFromFile2):",len(top10PostingDictPoppedAT20PercentFromFile2)
print "len(top10PostingDictPoppedAT30PercentFromFile2):",len(top10PostingDictPoppedAT30PercentFromFile2)
print "len(top10PostingDictPoppedAT40PercentFromFile2):",len(top10PostingDictPoppedAT40PercentFromFile2)
print "len(top10PostingDictPoppedAT50PercentFromFile2):",len(top10PostingDictPoppedAT50PercentFromFile2)
print "len(top10PostingDictPoppedAT60PercentFromFile2):",len(top10PostingDictPoppedAT60PercentFromFile2)
print "len(top10PostingDictPoppedAT70PercentFromFile2):",len(top10PostingDictPoppedAT70PercentFromFile2)
print "len(top10PostingDictPoppedAT80PercentFromFile2):",len(top10PostingDictPoppedAT80PercentFromFile2)
print "len(top10PostingDictPoppedAT90PercentFromFile2):",len(top10PostingDictPoppedAT90PercentFromFile2)
print "len(top10PostingDictPoppedAT100PercentFromFile2):",len(top10PostingDictPoppedAT100PercentFromFile2)
'''

print "Program Ends."
